# -*- coding:utf-8 -*-

__author__='zhaoxp'

import os
import os.path
import thread
import threading

'''
以@classmethod形式的多态去通用的构建对象
take mapreduce as an example
'''


class InputData(object):
    def read(self):
        raise NotImplementedError


class PathInputData(InputData):

    def __init__(self,path):
        super(PathInputData,self).__init__()
        self.path=path

    def read(self):
        return open(self.path).read()


class Worker(object):

    def __init__(self,input_data):
        self.input_data = input_data
        self.result = None

    def map(self):
        raise NotImplementedError

    def reduce(self,other):
        raise NotImplementedError


class LineCountWorker(Worker):

    def map(self):
        self.result = self.input_data.read().count('\n')

    def reduce(self, other):
        self.result += other.result


def generate_inputs(data_dir):
    for name in os.listdir(data_dir):
        if os.path.isfile(os.path.join(data_dir,name)) and name[-3:] == '.py':
            print(' ---1--- %s'%name)
            yield PathInputData(os.path.join(data_dir,name))



def create_workers(inputs):
    return [LineCountWorker(input_data) for input_data in inputs]
    '''
    workers = []
    for input_data in inputs:
        workers.append(LineCountWorker(input_data))
    return workers
    '''


def execute(workers):
    threads = [threading.Thread(target = worker.map) for worker in workers]
    for thread in threads: thread.start()
    for thread in threads: thread.join()

    first,rest = workers[0], workers[1:]
    for worker in rest:
        first.reduce(worker)
    return first.result


def mapreduce(data_dir):
    inputs = generate_inputs(data_dir)
    workers = create_workers(inputs)
    return execute(workers)


######################################################################################

class GenericInputData(object):

    def __init__(self,path):
        self.path = path

    def read(self):
        raise NotImplementedError

    @classmethod
    def generate_inputs(cls,config):
        raise NotImplementedError

class PathInputData2(GenericInputData):

    def read(self):
        return open(self.path).read()

    @classmethod
    def generate_inputs(cls,config):
        data_dir=config['data_dir']
        for name in os.listdir(data_dir):
            if os.path.isfile(os.path.join(data_dir,name)) and name[-3:] == '.py':
                print(' ---2--- %s'%name)
                yield cls(os.path.join(data_dir,name))


class GenericWorker(object):

    def __init__(self,input_data):
        self.input_data = input_data
        self.result = 0

    def map(self):
        raise NotImplementedError

    def reduce(self,other):
        raise NotImplementedError

    @classmethod
    def create_workers(cls,input_class,config):
        workers = []
        for input_data in input_class.generate_inputs(config):
            workers.append(cls(input_data))
        return workers


class LineCounterWorker2(GenericWorker):

    def map(self):
        self.result = self.input_data.read().count('\n')

    def reduce(self,other):
        self.result += other.result


def mapreduce2(worker_class,input_class,config):
    workers = worker_class.create_workers(input_class,config)
    return execute(workers)


if __name__=='__main__':
    print('----------old method:')
    result = mapreduce('d:\\temp\\py')
    print('result = %s'%result)
    print('----------new method:')
    config = {'data_dir':'D:\\temp\\py'}
    result2 = mapreduce2(LineCounterWorker2, PathInputData2, config)
    print('result2 = %s'%result2)